{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据集下载\n",
    "\n",
    "使用THUCNews的一个子集进行训练与测试：[https://www.lanzous.com/i5t0lsd](https://www.lanzous.com/i5t0lsd)\n",
    "\n",
    "本次训练使用了其中的10个分类，每个分类6500条数据。类别如下：\n",
    "\n",
    "体育, 财经, 房产, 家居, 教育, 科技, 时尚, 时政, 游戏, 娱乐\n",
    "\n",
    "### cnews_loader.py为数据的预处理文件。\n",
    "\n",
    "- read_file(): 读取文件数据;\n",
    "- build_vocab(): 构建词汇表，使用字符级的表示，这一函数会将词汇表存储下来，避免每一次重复处理;\n",
    "- read_vocab(): 读取上一步存储的词汇表，转换为{词：id}表示;\n",
    "- read_category(): 将分类目录固定，转换为{类别: id}表示;\n",
    "- to_words(): 将一条由id表示的数据重新转换为文字;\n",
    "- process_file(): 将数据集从文字转换为固定长度的id序列表示;\n",
    "- batch_iter(): 为神经网络的训练准备经过shuffle的批次的数据。\n",
    "\n",
    "### textCNN的模型和可配置的参数，在cnn_model.py中。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from datetime import timedelta\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn import metrics\n",
    "\n",
    "from cnn_model import TCNNConfig, TextCNN\n",
    "from cnews_loader import read_vocab, read_category, batch_iter, process_file, build_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = 'cnews'\n",
    "train_dir = os.path.join(base_dir, 'cnews.train.txt')\n",
    "test_dir = os.path.join(base_dir, 'cnews.test.txt')\n",
    "val_dir = os.path.join(base_dir, 'cnews.val.txt')\n",
    "vocab_dir = os.path.join(base_dir, 'cnews.vocab.txt')\n",
    "\n",
    "save_dir = 'checkpoints/textcnn'\n",
    "save_path = os.path.join(save_dir, 'best_validation')  # 最佳验证结果保存路径\n",
    "\n",
    "\n",
    "def get_time_dif(start_time):\n",
    "    \"\"\"获取已使用时间\"\"\"\n",
    "    end_time = time.time()\n",
    "    time_dif = end_time - start_time\n",
    "    return timedelta(seconds=int(round(time_dif)))\n",
    "\n",
    "\n",
    "def feed_data(x_batch, y_batch, keep_prob):\n",
    "    feed_dict = {\n",
    "        model.input_x: x_batch,\n",
    "        model.input_y: y_batch,\n",
    "        model.keep_prob: keep_prob\n",
    "    }\n",
    "    return feed_dict\n",
    "\n",
    "\n",
    "def evaluate(sess, x_, y_):\n",
    "    \"\"\"评估在某一数据上的准确率和损失\"\"\"\n",
    "    data_len = len(x_)\n",
    "    batch_eval = batch_iter(x_, y_, 128)\n",
    "    total_loss = 0.0\n",
    "    total_acc = 0.0\n",
    "    for x_batch, y_batch in batch_eval:\n",
    "        batch_len = len(x_batch)\n",
    "        feed_dict = feed_data(x_batch, y_batch, 1.0)\n",
    "        loss, acc = sess.run([model.loss, model.acc], feed_dict=feed_dict)\n",
    "        total_loss += loss * batch_len\n",
    "        total_acc += acc * batch_len\n",
    "\n",
    "    return total_loss / data_len, total_acc / data_len\n",
    "\n",
    "\n",
    "def train():\n",
    "    print(\"Configuring TensorBoard and Saver...\")\n",
    "    # 配置 Tensorboard，重新训练时，请将tensorboard文件夹删除，不然图会覆盖\n",
    "    tensorboard_dir = 'tensorboard/textcnn'\n",
    "    if not os.path.exists(tensorboard_dir):\n",
    "        os.makedirs(tensorboard_dir)\n",
    "\n",
    "    tf.summary.scalar(\"loss\", model.loss)\n",
    "    tf.summary.scalar(\"accuracy\", model.acc)\n",
    "    merged_summary = tf.summary.merge_all()\n",
    "    writer = tf.summary.FileWriter(tensorboard_dir)\n",
    "\n",
    "    # 配置 Saver\n",
    "    saver = tf.train.Saver()\n",
    "    if not os.path.exists(save_dir):\n",
    "        os.makedirs(save_dir)\n",
    "\n",
    "    print(\"Loading training and validation data...\")\n",
    "    # 载入训练集与验证集\n",
    "    start_time = time.time()\n",
    "    x_train, y_train = process_file(train_dir, word_to_id, cat_to_id, config.seq_length)\n",
    "    x_val, y_val = process_file(val_dir, word_to_id, cat_to_id, config.seq_length)\n",
    "    time_dif = get_time_dif(start_time)\n",
    "    print(\"Time usage:\", time_dif)\n",
    "\n",
    "    # 创建session\n",
    "    session = tf.Session()\n",
    "    session.run(tf.global_variables_initializer())\n",
    "    writer.add_graph(session.graph)\n",
    "\n",
    "    print('Training and evaluating...')\n",
    "    start_time = time.time()\n",
    "    total_batch = 0  # 总批次\n",
    "    best_acc_val = 0.0  # 最佳验证集准确率\n",
    "    last_improved = 0  # 记录上一次提升批次\n",
    "    require_improvement = 1000  # 如果超过1000轮未提升，提前结束训练\n",
    "\n",
    "    flag = False\n",
    "    for epoch in range(config.num_epochs):\n",
    "        print('Epoch:', epoch + 1)\n",
    "        batch_train = batch_iter(x_train, y_train, config.batch_size)\n",
    "        for x_batch, y_batch in batch_train:\n",
    "            feed_dict = feed_data(x_batch, y_batch, config.dropout_keep_prob)\n",
    "\n",
    "            if total_batch % config.save_per_batch == 0:\n",
    "                # 每多少轮次将训练结果写入tensorboard scalar\n",
    "                s = session.run(merged_summary, feed_dict=feed_dict)\n",
    "                writer.add_summary(s, total_batch)\n",
    "\n",
    "            if total_batch % config.print_per_batch == 0:\n",
    "                # 每多少轮次输出在训练集和验证集上的性能\n",
    "                feed_dict[model.keep_prob] = 1.0\n",
    "                loss_train, acc_train = session.run([model.loss, model.acc], feed_dict=feed_dict)\n",
    "                loss_val, acc_val = evaluate(session, x_val, y_val)  # todo\n",
    "\n",
    "                if acc_val > best_acc_val:\n",
    "                    # 保存最好结果\n",
    "                    best_acc_val = acc_val\n",
    "                    last_improved = total_batch\n",
    "                    saver.save(sess=session, save_path=save_path)\n",
    "                    improved_str = '*'\n",
    "                else:\n",
    "                    improved_str = ''\n",
    "\n",
    "                time_dif = get_time_dif(start_time)\n",
    "                msg = 'Iter: {0:>6}, Train Loss: {1:>6.2}, Train Acc: {2:>7.2%},' \\\n",
    "                      + ' Val Loss: {3:>6.2}, Val Acc: {4:>7.2%}, Time: {5} {6}'\n",
    "                print(msg.format(total_batch, loss_train, acc_train, loss_val, acc_val, time_dif, improved_str))\n",
    "\n",
    "            feed_dict[model.keep_prob] = config.dropout_keep_prob\n",
    "            session.run(model.optim, feed_dict=feed_dict)  # 运行优化\n",
    "            total_batch += 1\n",
    "\n",
    "            if total_batch - last_improved > require_improvement:\n",
    "                # 验证集正确率长期不提升，提前结束训练\n",
    "                print(\"No optimization for a long time, auto-stopping...\")\n",
    "                flag = True\n",
    "                break  # 跳出循环\n",
    "        if flag:  # 同上\n",
    "            break\n",
    "\n",
    "\n",
    "def test():\n",
    "    print(\"Loading test data...\")\n",
    "    start_time = time.time()\n",
    "    x_test, y_test = process_file(test_dir, word_to_id, cat_to_id, config.seq_length)\n",
    "\n",
    "    session = tf.Session()\n",
    "    session.run(tf.global_variables_initializer())\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess=session, save_path=save_path)  # 读取保存的模型\n",
    "\n",
    "    print('Testing...')\n",
    "    loss_test, acc_test = evaluate(session, x_test, y_test)\n",
    "    msg = 'Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}'\n",
    "    print(msg.format(loss_test, acc_test))\n",
    "\n",
    "    batch_size = 128\n",
    "    data_len = len(x_test)\n",
    "    num_batch = int((data_len - 1) / batch_size) + 1\n",
    "\n",
    "    y_test_cls = np.argmax(y_test, 1)\n",
    "    y_pred_cls = np.zeros(shape=len(x_test), dtype=np.int32)  # 保存预测结果\n",
    "    for i in range(num_batch):  # 逐批次处理\n",
    "        start_id = i * batch_size\n",
    "        end_id = min((i + 1) * batch_size, data_len)\n",
    "        feed_dict = {\n",
    "            model.input_x: x_test[start_id:end_id],\n",
    "            model.keep_prob: 1.0\n",
    "        }\n",
    "        y_pred_cls[start_id:end_id] = session.run(model.y_pred_cls, feed_dict=feed_dict)\n",
    "\n",
    "    # 评估\n",
    "    print(\"Precision, Recall and F1-Score...\")\n",
    "    print(metrics.classification_report(y_test_cls, y_pred_cls, target_names=categories))\n",
    "\n",
    "    # 混淆矩阵\n",
    "    print(\"Confusion Matrix...\")\n",
    "    cm = metrics.confusion_matrix(y_test_cls, y_pred_cls)\n",
    "    print(cm)\n",
    "\n",
    "    time_dif = get_time_dif(start_time)\n",
    "    print(\"Time usage:\", time_dif)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 开始训练\n",
    "\n",
    "sys.argv 输出运行文件名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0826 20:03:59.630935 140222699312960 module_wrapper.py:136] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/util/module_wrapper.py:163: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Configuring CNN model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0826 20:03:59.934021 140222699312960 deprecation.py:323] From /home/python_home/WeiZhongChuang/ML/RNN/cnn_model.py:50: conv1d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv1D` instead.\n",
      "W0826 20:03:59.935921 140222699312960 deprecation.py:323] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/layers/convolutional.py:218: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "W0826 20:04:00.065614 140222699312960 deprecation.py:323] From /home/python_home/WeiZhongChuang/ML/RNN/cnn_model.py:56: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "W0826 20:04:00.080801 140222699312960 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W0826 20:04:00.126461 140222699312960 deprecation.py:323] From /home/python_home/WeiZhongChuang/ML/RNN/cnn_model.py:66: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "W0826 20:04:00.317426 140222699312960 module_wrapper.py:136] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/util/module_wrapper.py:163: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "W0826 20:04:00.534005 140222699312960 module_wrapper.py:136] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/util/module_wrapper.py:163: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Configuring TensorBoard and Saver...\n",
      "Loading training and validation data...\n",
      "Time usage: 0:00:13\n",
      "Training and evaluating...\n",
      "Epoch: 1\n",
      "Iter:      0, Train Loss:    2.3, Train Acc:  10.94%, Val Loss:    2.3, Val Acc:  10.00%, Time: 0:00:04 *\n",
      "Iter:    100, Train Loss:   0.83, Train Acc:  76.56%, Val Loss:    1.2, Val Acc:  66.44%, Time: 0:00:24 *\n",
      "Iter:    200, Train Loss:    0.2, Train Acc:  93.75%, Val Loss:   0.67, Val Acc:  78.18%, Time: 0:00:43 *\n",
      "Iter:    300, Train Loss:   0.14, Train Acc:  95.31%, Val Loss:   0.42, Val Acc:  87.84%, Time: 0:01:04 *\n",
      "Iter:    400, Train Loss:   0.12, Train Acc:  96.88%, Val Loss:   0.37, Val Acc:  90.06%, Time: 0:01:25 *\n",
      "Iter:    500, Train Loss:   0.16, Train Acc:  96.88%, Val Loss:   0.35, Val Acc:  89.48%, Time: 0:01:46 \n",
      "Iter:    600, Train Loss:    0.2, Train Acc:  92.19%, Val Loss:   0.26, Val Acc:  92.38%, Time: 0:02:08 *\n",
      "Iter:    700, Train Loss:   0.13, Train Acc:  96.88%, Val Loss:   0.27, Val Acc:  92.50%, Time: 0:02:30 *\n",
      "Epoch: 2\n",
      "Iter:    800, Train Loss:   0.14, Train Acc:  95.31%, Val Loss:   0.24, Val Acc:  93.20%, Time: 0:02:52 *\n",
      "Iter:    900, Train Loss:  0.061, Train Acc:  98.44%, Val Loss:   0.25, Val Acc:  92.98%, Time: 0:03:14 \n",
      "Iter:   1000, Train Loss:  0.038, Train Acc:  98.44%, Val Loss:   0.24, Val Acc:  92.70%, Time: 0:03:35 \n",
      "Iter:   1100, Train Loss:   0.22, Train Acc:  93.75%, Val Loss:   0.24, Val Acc:  93.82%, Time: 0:03:57 *\n",
      "Iter:   1200, Train Loss:  0.077, Train Acc:  96.88%, Val Loss:    0.2, Val Acc:  94.24%, Time: 0:04:19 *\n",
      "Iter:   1300, Train Loss:  0.098, Train Acc:  98.44%, Val Loss:   0.24, Val Acc:  92.90%, Time: 0:04:40 \n",
      "Iter:   1400, Train Loss:  0.064, Train Acc:  96.88%, Val Loss:   0.19, Val Acc:  94.68%, Time: 0:05:02 *\n",
      "Iter:   1500, Train Loss:   0.12, Train Acc:  95.31%, Val Loss:   0.21, Val Acc:  94.22%, Time: 0:05:24 \n",
      "Epoch: 3\n",
      "Iter:   1600, Train Loss:  0.079, Train Acc:  98.44%, Val Loss:   0.22, Val Acc:  93.52%, Time: 0:05:46 \n",
      "Iter:   1700, Train Loss:  0.083, Train Acc:  98.44%, Val Loss:   0.18, Val Acc:  95.00%, Time: 0:06:09 *\n",
      "Iter:   1800, Train Loss:    0.2, Train Acc:  98.44%, Val Loss:   0.22, Val Acc:  93.70%, Time: 0:06:31 \n",
      "Iter:   1900, Train Loss:   0.17, Train Acc:  98.44%, Val Loss:   0.25, Val Acc:  92.82%, Time: 0:06:53 \n",
      "Iter:   2000, Train Loss:  0.029, Train Acc: 100.00%, Val Loss:    0.2, Val Acc:  94.26%, Time: 0:07:16 \n",
      "Iter:   2100, Train Loss:   0.16, Train Acc:  93.75%, Val Loss:   0.19, Val Acc:  94.92%, Time: 0:07:38 \n",
      "Iter:   2200, Train Loss:  0.019, Train Acc: 100.00%, Val Loss:   0.19, Val Acc:  94.86%, Time: 0:08:00 \n",
      "Iter:   2300, Train Loss:  0.038, Train Acc:  98.44%, Val Loss:   0.19, Val Acc:  95.02%, Time: 0:08:23 *\n",
      "Epoch: 4\n",
      "Iter:   2400, Train Loss:  0.069, Train Acc:  96.88%, Val Loss:   0.23, Val Acc:  94.20%, Time: 0:08:46 \n",
      "Iter:   2500, Train Loss:   0.03, Train Acc:  98.44%, Val Loss:   0.18, Val Acc:  95.40%, Time: 0:09:08 *\n",
      "Iter:   2600, Train Loss:  0.046, Train Acc:  98.44%, Val Loss:   0.21, Val Acc:  94.30%, Time: 0:09:31 \n",
      "Iter:   2700, Train Loss: 0.0095, Train Acc: 100.00%, Val Loss:   0.18, Val Acc:  95.10%, Time: 0:09:54 \n",
      "Iter:   2800, Train Loss:  0.075, Train Acc:  96.88%, Val Loss:   0.18, Val Acc:  95.48%, Time: 0:10:17 *\n",
      "Iter:   2900, Train Loss:  0.064, Train Acc:  96.88%, Val Loss:   0.18, Val Acc:  95.12%, Time: 0:10:40 \n",
      "Iter:   3000, Train Loss: 0.0075, Train Acc: 100.00%, Val Loss:    0.2, Val Acc:  94.98%, Time: 0:11:02 \n",
      "Iter:   3100, Train Loss:  0.094, Train Acc:  96.88%, Val Loss:   0.18, Val Acc:  95.64%, Time: 0:11:25 *\n",
      "Epoch: 5\n",
      "Iter:   3200, Train Loss: 0.0053, Train Acc: 100.00%, Val Loss:   0.19, Val Acc:  95.52%, Time: 0:11:48 \n",
      "Iter:   3300, Train Loss: 0.0098, Train Acc: 100.00%, Val Loss:    0.2, Val Acc:  95.04%, Time: 0:12:11 \n",
      "Iter:   3400, Train Loss:  0.028, Train Acc:  98.44%, Val Loss:   0.17, Val Acc:  95.72%, Time: 0:12:35 *\n",
      "Iter:   3500, Train Loss: 0.0095, Train Acc: 100.00%, Val Loss:   0.26, Val Acc:  93.20%, Time: 0:12:58 \n",
      "Iter:   3600, Train Loss:  0.045, Train Acc:  98.44%, Val Loss:   0.17, Val Acc:  95.56%, Time: 0:13:21 \n",
      "Iter:   3700, Train Loss:  0.013, Train Acc: 100.00%, Val Loss:   0.21, Val Acc:  94.58%, Time: 0:13:44 \n",
      "Iter:   3800, Train Loss: 0.0057, Train Acc: 100.00%, Val Loss:   0.17, Val Acc:  95.52%, Time: 0:14:08 \n",
      "Iter:   3900, Train Loss: 0.0091, Train Acc: 100.00%, Val Loss:   0.19, Val Acc:  95.58%, Time: 0:14:31 \n",
      "Epoch: 6\n",
      "Iter:   4000, Train Loss: 0.0015, Train Acc: 100.00%, Val Loss:    0.2, Val Acc:  95.12%, Time: 0:14:54 \n",
      "Iter:   4100, Train Loss: 0.0022, Train Acc: 100.00%, Val Loss:   0.19, Val Acc:  95.32%, Time: 0:15:18 \n",
      "Iter:   4200, Train Loss: 0.0091, Train Acc: 100.00%, Val Loss:    0.2, Val Acc:  95.46%, Time: 0:15:41 \n",
      "Iter:   4300, Train Loss: 0.00033, Train Acc: 100.00%, Val Loss:   0.24, Val Acc:  94.84%, Time: 0:16:04 \n",
      "Iter:   4400, Train Loss:   0.01, Train Acc: 100.00%, Val Loss:   0.22, Val Acc:  94.52%, Time: 0:16:28 \n",
      "No optimization for a long time, auto-stopping...\n"
     ]
    }
   ],
   "source": [
    "type_ = 'train'\n",
    "\n",
    "print('Configuring CNN model...')\n",
    "config = TCNNConfig()\n",
    "if not os.path.exists(vocab_dir):  # 如果不存在词汇表，重建\n",
    "    build_vocab(train_dir, vocab_dir, config.vocab_size)\n",
    "categories, cat_to_id = read_category()\n",
    "words, word_to_id = read_vocab(vocab_dir)\n",
    "config.vocab_size = len(words)\n",
    "model = TextCNN(config)\n",
    "\n",
    "if type_ == 'train':\n",
    "    train()\n",
    "else:\n",
    "    test()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading test data...\n",
      "Testing...\n",
      "Test Loss:   0.11, Test Acc:  97.05%\n",
      "Precision, Recall and F1-Score...\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          体育       1.00      0.99      0.99      1000\n",
      "          财经       0.98      0.98      0.98      1000\n",
      "          房产       1.00      1.00      1.00      1000\n",
      "          家居       0.97      0.92      0.94      1000\n",
      "          教育       0.93      0.96      0.95      1000\n",
      "          科技       0.95      0.98      0.96      1000\n",
      "          时尚       0.98      0.97      0.97      1000\n",
      "          时政       0.95      0.96      0.95      1000\n",
      "          游戏       0.97      0.98      0.98      1000\n",
      "          娱乐       0.98      0.97      0.98      1000\n",
      "\n",
      "    accuracy                           0.97     10000\n",
      "   macro avg       0.97      0.97      0.97     10000\n",
      "weighted avg       0.97      0.97      0.97     10000\n",
      "\n",
      "Confusion Matrix...\n",
      "[[990   0   0   0   3   3   0   2   1   1]\n",
      " [  0 977   0   2   5   4   0  12   0   0]\n",
      " [  0   0 996   4   0   0   0   0   0   0]\n",
      " [  0   9   0 916  26  11   5  27   4   2]\n",
      " [  0   2   0   6 958  12   6   8   4   4]\n",
      " [  0   0   0   2   4 979   2   1  11   1]\n",
      " [  1   0   0   4   2   7 972   1   6   7]\n",
      " [  0   7   0   4  17  11   0 960   1   0]\n",
      " [  1   1   0   1   3   2   5   0 984   3]\n",
      " [  1   0   0   4   8   4   4   1   5 973]]\n",
      "Time usage: 0:00:15\n"
     ]
    }
   ],
   "source": [
    "\n",
    "test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
